﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Projekt
{
    class Tokenizer
    {

        /// <summary>
        /// Method to tokenize text
        /// </summary>
        /// <param name="text">string - Text to tokenize</param>
        /// <returns>string - Tokenized text</returns>
        public string tokenize(string text)
        {
            // Put whitespace around special char
            Regex regexSpecialChar = new Regex(@"([^\w\d\s])");
            MatchEvaluator regexSpecialCharEvaluator = delegate( Match match ) 
            {
                return " " + match.Groups[1].ToString() + " ";
            };
            text = regexSpecialChar.Replace(text, regexSpecialCharEvaluator);

            // Replace all whitespaces with linebreak
            Regex regexToken = new Regex(@"(\s+)");
            string tokenizedString = regexToken.Replace(text, "\n");
            return tokenizedString;
        }

    }
}
